import 

def cmpval_item_rev(x,y) :
  return y[1] - x[1]

list_ponctuation = ['.', ',', ';', '!', '?', '(', ')', '[', ']']

def clean_space(str_unicode) : 
  return re.sub(u'[\s]+', u' ', str_unicode)

def clean_ponctuation(str_unicode, list_ponctuation) :
  for p in list_ponctuation :
    s =  '%s[\s]'%(p)
    str_unicode = re.sub(u'%s[\s]'%(p), u' %s '%(p), str_unicode)
  return str_unicode  

def list_words2zipf(list_words) :
  res = {}
  for w in list_words :
    tm.dict_count(res, w)
  it = res.items()
  it.sort(cmpval_item_rev)
  list_x = [elt[1] for elt in it]
  list_y = range(1, len(list_x)+1)
  return {'list_x' : list_x, 'list_y' : list_y}

def list_words2zipf_log(list_words) :
  res = {}
  for w in list_words :
    tm.dict_count(res, w)
  it = res.items()
  it.sort(cmpval_item_rev)
  list_x = [math.log(elt[1]) for elt in it]
  list_y = [math.log(i) for i in xrange(1, len(list_x)+1)]
  return {'list_x' : list_x, 'list_y' : list_y}

def zipf2zipf_log(zipf) :
  list_x = [math.log(elt) for elt in zipf['list_x']]
  list_y = [math.log(i) for i in xrange(1, len(list_x)+1)]
  return {'list_x' : list_x, 'list_y' : list_y}
